"""
Case Type   : 资源池化
Case Name   : 备机有业务场景下支持在线failover，并行回放模式下，升主节点网络中断
Create At   : 2025.04
Owner       : haomeng
Description :
    1、查看集群状态 cm_ctl query -Cv
    2、查看回放参数recovery_max_workers
    3、主节点执行tpcc读写业务，备节点执行tpcc只读业务
    4、主节点mv掉dn实例路径，kill掉dn实例进程，构造failover场景
    5、中断待升主节点网络，再次构造failover场景
    6、恢复环境，恢复集群状态
        cm_ctl start -n 1
        cm_ctl switchover -a
Expect      :
    1、集群状态正常，数据库节点状态正常
    2、默认值为4
    3、业务执行成功
    4、主节点异常,集群状态异常,dn进程不存在,任意备节点升主，且升主节点未发生重启failover
    5、升主节点异常被踢出，剩余备节点升主，且升主节点未发生重启failover
    6、拉起节点成功,重启集群成功,恢复集群初始状态成功
History     :
"""

import os
import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.CommonSH import CommonSH
from testcase.utils.Common import Common
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node
from yat.test import macro

Primary_SH = CommonSH('PrimaryDbUser')


@unittest.skipIf(1 == Primary_SH.get_node_num(), '单机环境不执行')
class SharedStorage(unittest.TestCase):

    def setUp(self):
        self.log = Logger()
        self.log.info(f'-----{os.path.basename(__file__)} start-----')
        self.constant = Constant()
        self.com = Common()
        self.root_usernode = [Node(n) for n in ('PrimaryRoot', 'Standby1Root', 'Standby2Root')]
        self.nodes = [Node(n) for n in ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')]
        self.bak_path = os.path.join(os.path.dirname(macro.DB_INSTANCE_PATH), 'dn_bak')
        self.param = 'recovery_max_workers'

        self.log.info("-------------获取网卡名---------------")
        self.networkcard = []
        network_card_cmd = 'ifconfig'
        self.log.info(network_card_cmd)
        for i in range(3):
            network_card_result = self.root_usernode[i].sh(
                network_card_cmd).result()
            network_card_list = network_card_result.split('\n\n')
            self.log.info(network_card_list)        
            for network_card_item in network_card_list:
                if self.nodes[i].db_host in network_card_item:
                    current_network = network_card_item.split(':')[0].strip()
                    self.networkcard.append(current_network) 
                    self.log.info(self.networkcard)

    def test_shared_storage(self):
        text = '-----step1:查看集群状态; expect:集群状态正常-----'
        self.log.info(text)

        status = Primary_SH.exec_cm_ctl(mode='query', param='-Cv')
        self.log.info(status)
        self.assertIn('cluster_state   : Normal', status, '执行失败' + text)

        text = '-----step2:查看回放参数  expect:应为默认值4，否则修改为4-----'
        self.log.info(text)

        self.default = self.com.show_param(self.param)
        self.log.info(self.default)
        if self.default != '4':
            Primary_SH.execute_gsguc('set', self.constant.GSGUC_SUCCESS_MSG,
                                     f'{self.param} = 4')
            Primary_SH.restart_db_cluster()
            param_value = self.com.show_param(self.param)
            self.assertEqual('4', param_value, f'{self.param}修改失败')
        else:
            self.assertEqual('4', self.default, f'{self.param}默认值正常')

        text = '-----step3:主节点执行读写业务，备节点执行只读业务  expect:启动业务成功-----'
        self.log.info(text)

        tpcc_cmd_p = f'. runBenchmark.sh {macro.PRI_PROPS}'
        tpcc_thread_p = ComThread(self.com.startTPCC,
                                  args=(self.root_usernode[2],
                                        macro.TPCC_PATH,
                                        tpcc_cmd_p,))
        tpcc_thread_p.setDaemon(True)
        tpcc_thread_p.start()

        tpcc_cmd_s = f'. runBenchmark.sh {macro.STA_PROPS}'
        tpcc_thread_s = ComThread(self.com.startTPCC,
                                  args=(self.root_usernode[2],
                                        macro.TPCC_PATH,
                                        tpcc_cmd_s,))
        tpcc_thread_s.setDaemon(True)
        tpcc_thread_s.start()

        text = '-----step4:移除主节点dn路径; expect:停止成功-----'
        self.log.info(text)

        mv_dn = f'''mv {macro.DB_INSTANCE_PATH} {self.bak_path};
                    ls {self.bak_path}'''
        dn_replace = self.nodes[0].sh(mv_dn).result()
        self.log.info(dn_replace)
        self.assertIn('postgresql.conf', dn_replace, '执行失败' + text)

        self.log.info('-----查看主节点dn进程并杀死进程,查看集群状态-----')
        dn_pid1 = []
        for n in range(3):
            pid = self.com.get_pid(self.nodes[n], 'bin/gaussdb')
            dn_pid1.append(pid)
        self.log.info(dn_pid1)

        self.nodes[0].sh(f'''kill -9 {dn_pid1[0]}''').result()

        time.sleep(120)
        sta1 = Primary_SH.exec_cm_ctl('query', '-Cv')
        self.log.info(sta1)
        self.assertIn('cluster_state   : Degraded', sta1, '执行失败' + text)
        self.assertIn('P Down', sta1, '主节点正常')
        self.assertIn('S Primary Normal', sta1, '未有备节点升为主节点')

        dn_pid2 = []
        for n in range(1, 3):
            pid = self.com.get_pid(self.nodes[n], 'bin/gaussdb')
            dn_pid2.append(pid)
        self.assertEqual(dn_pid1[1], dn_pid2[0], 'dn进程发生改变')
        self.assertEqual(dn_pid1[2], dn_pid2[1], 'dn进程发生改变')

        text = '----step5: 中断待升主节点网络，再次构造failover场景 expect:成功----'
        self.log.info(text)

        self.log.info('-----获取当前主节点的节点ID-----')
        p_status = sta1.splitlines()[-1].split('|')
        for i in p_status:
            if 'S Primary' in i or 'Promoting' in i:
                self.current_p_id = int(i.split()[0])
                self.log.info(f'{self.current_p_id}节点升为主节点')

        inject_command = f"rNet_down (dev, time)" \
                         f"values({self.networkcard[int(self.current_p_id-1)]}, 60)"
        cfe_thread = ComThread(
            self.com.cfe_inject, args=(self.root_usernode[int(self.current_p_id-1)],
                                       inject_command,))
        cfe_thread.setDaemon(True)
        cfe_thread.start()
        cfe_thread.join(60)
        result = cfe_thread.get_result()
        self.log.info(result)
        self.assertIn(self.constant.cfe_inject_netdown_success_msg, result,
                      '执行失败' + text)

        self.log.info('-----查看各节点dn进程,查看集群状态-----')
        dn_pid1 = []
        for n in range(3):
            pid = self.com.get_pid(self.nodes[n], 'bin/gaussdb')
            dn_pid1.append(pid)
        self.log.info(dn_pid1)

        time.sleep(120)
        sta1 = Primary_SH.exec_cm_ctl('query', '-Cv')
        self.log.info(sta1)
        self.assertNotIn('cluster_state   : Normal', sta1, '执行失败' + text)
        self.assertIn('P Down', sta1, '主节点正常')
        self.assertTrue('S Primary' in sta1 or 'Promoting' in sta1, '未有备节点升为主节点')

        dn_pid2 = []
        for n in range(1, 3):
            pid = self.com.get_pid(self.nodes[n], 'bin/gaussdb')
            dn_pid2.append(pid)
        self.assertEqual(dn_pid1[2], dn_pid2[1], 'dn进程发生改变')

        tpcc_thread_p.join(70)
        p_res = tpcc_thread_p.get_result()
        self.log.info(p_res)

        tpcc_thread_s.join(70)
        s_res = tpcc_thread_s.get_result()
        self.log.info(s_res)

    def tearDown(self):
        text = '-----step6:恢复集群,恢复集群初始状态;  expect:启动成功,集群状态正常-----'
        self.log.info(text)

        remv_dn = f'''mv {self.bak_path} {macro.DB_INSTANCE_PATH};
                      ls {macro.DB_INSTANCE_PATH}'''
        dn_replace = self.nodes[0].sh(remv_dn).result()
        self.log.info(dn_replace)

        startdn = Primary_SH.exec_cm_ctl(mode='start')
        self.log.info(startdn)

        time.sleep(60)
        recovery = Primary_SH.exec_cm_ctl(mode='switchover', param='-a')
        self.log.info(recovery)

        status = Primary_SH.exec_cm_ctl(mode='query', param='-Cv')
        self.log.info(status)

        self.assertIn('postgresql.conf', dn_replace, '恢复dn失败')
        self.assertIn(self.constant.cm_start_success_msg, startdn, '重启集群成功')
        self.assertIn(self.constant.cm_switchover_success_msg, recovery,
                      '执行失败' + text)
        self.assertIn('cluster_state   : Normal', status, '执行失败' + text)
        self.assertIn('P Primary Normal',
                      status.splitlines()[-1].split('|')[0].split('6001')[-1],
                      '执行失败' + text)
        self.assertIn('S Standby Normal',
                      status.splitlines()[-1].split('|')[1].split('6002')[-1],
                      '执行失败' + text)
        self.log.info(f'-----{os.path.basename(__file__)} end-----')
